{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ef08f1e9-2af5-4b3b-be45-7f9123dae90c",
   "metadata": {},
   "source": [
    "# Babyweight Data Preprocessing with tf.Transform"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7a716bc-ce8d-4f29-a9d2-c83371137b81",
   "metadata": {},
   "source": [
    "## 1. Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80870765-75f7-4bce-a49f-d6dcdc76b066",
   "metadata": {},
   "source": [
    "### Install required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "adef56d2-e756-441a-b7f2-907a67b51b47",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: apache-beam==2.46.0 in /opt/conda/lib/python3.7/site-packages (2.46.0)\n",
      "Requirement already satisfied: pyarrow<10.0.0,>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (9.0.0)\n",
      "Requirement already satisfied: grpcio!=1.48.0,<2,>=1.33.1 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (1.53.0)\n",
      "Requirement already satisfied: httplib2<0.22.0,>=0.8 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (0.21.0)\n",
      "Requirement already satisfied: proto-plus<2,>=1.7.1 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (1.22.2)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.24.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (2.28.2)\n",
      "Requirement already satisfied: numpy<1.25.0,>=1.14.3 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (1.21.6)\n",
      "Requirement already satisfied: objsize<0.7.0,>=0.6.1 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (0.6.1)\n",
      "Requirement already satisfied: regex>=2020.6.8 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (2022.10.31)\n",
      "Requirement already satisfied: dill<0.3.2,>=0.3.1.1 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (0.3.1.1)\n",
      "Requirement already satisfied: crcmod<2.0,>=1.7 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (1.7)\n",
      "Requirement already satisfied: pytz>=2018.3 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (2023.3)\n",
      "Requirement already satisfied: pydot<2,>=1.2.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (1.4.2)\n",
      "Requirement already satisfied: hdfs<3.0.0,>=2.1.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (2.7.0)\n",
      "Requirement already satisfied: cloudpickle~=2.2.1 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (2.2.1)\n",
      "Requirement already satisfied: python-dateutil<3,>=2.8.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (2.8.2)\n",
      "Requirement already satisfied: protobuf<4,>3.12.2 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (3.20.1)\n",
      "Requirement already satisfied: pymongo<4.0.0,>=3.8.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (3.13.0)\n",
      "Requirement already satisfied: fastavro<2,>=0.23.6 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (1.7.3)\n",
      "Requirement already satisfied: orjson<4.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (3.8.10)\n",
      "Requirement already satisfied: zstandard<1,>=0.18.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (0.18.0)\n",
      "Requirement already satisfied: fasteners<1.0,>=0.3 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (0.18)\n",
      "Requirement already satisfied: typing-extensions>=3.7.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam==2.46.0) (4.5.0)\n",
      "Requirement already satisfied: docopt in /opt/conda/lib/python3.7/site-packages (from hdfs<3.0.0,>=2.1.0->apache-beam==2.46.0) (0.6.2)\n",
      "Requirement already satisfied: six>=1.9.0 in /opt/conda/lib/python3.7/site-packages (from hdfs<3.0.0,>=2.1.0->apache-beam==2.46.0) (1.16.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 in /opt/conda/lib/python3.7/site-packages (from httplib2<0.22.0,>=0.8->apache-beam==2.46.0) (3.0.9)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0,>=2.24.0->apache-beam==2.46.0) (3.4)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0,>=2.24.0->apache-beam==2.46.0) (2.1.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0,>=2.24.0->apache-beam==2.46.0) (2022.12.7)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0,>=2.24.0->apache-beam==2.46.0) (1.26.15)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!sudo -i pip install apache-beam==2.46.0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0cbdad52-9615-4153-81f7-d1fff38b6cd0",
   "metadata": {},
   "source": [
    "You can ingnore the dependency resolver errors. Confirm the final message starting with \"Successfully installed ...\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "206794d5-3682-47e2-bd83-de167401be7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tensorflow-transform==1.6.0\n",
      "  Downloading tensorflow_transform-1.6.0-py3-none-any.whl (427 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m427.5/427.5 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: absl-py<2.0.0,>=0.9 in /opt/conda/lib/python3.7/site-packages (from tensorflow-transform==1.6.0) (1.4.0)\n",
      "Requirement already satisfied: numpy<2,>=1.16 in /opt/conda/lib/python3.7/site-packages (from tensorflow-transform==1.6.0) (1.21.6)\n",
      "Collecting tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5\n",
      "  Downloading tensorflow-2.7.4-cp37-cp37m-manylinux2010_x86_64.whl (495.5 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m495.5/495.5 MB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: apache-beam[gcp]<3,>=2.35 in /opt/conda/lib/python3.7/site-packages (from tensorflow-transform==1.6.0) (2.46.0)\n",
      "Collecting tensorflow-metadata<1.7.0,>=1.6.0\n",
      "  Downloading tensorflow_metadata-1.6.0-py3-none-any.whl (48 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m48.8/48.8 kB\u001b[0m \u001b[31m283.9 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: protobuf<4,>=3.13 in /opt/conda/lib/python3.7/site-packages (from tensorflow-transform==1.6.0) (3.20.1)\n",
      "Collecting pyarrow<6,>=1\n",
      "  Downloading pyarrow-5.0.0-cp37-cp37m-manylinux2014_x86_64.whl (23.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.6/23.6 MB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting tfx-bsl<1.7.0,>=1.6.0\n",
      "  Downloading tfx_bsl-1.6.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (19.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.1/19.1 MB\u001b[0m \u001b[31m20.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pydot<2,>=1.2 in /opt/conda/lib/python3.7/site-packages (from tensorflow-transform==1.6.0) (1.4.2)\n",
      "Requirement already satisfied: fasteners<1.0,>=0.3 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (0.18)\n",
      "Requirement already satisfied: hdfs<3.0.0,>=2.1.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (2.7.0)\n",
      "Requirement already satisfied: python-dateutil<3,>=2.8.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (2.8.2)\n",
      "Requirement already satisfied: objsize<0.7.0,>=0.6.1 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (0.6.1)\n",
      "Requirement already satisfied: regex>=2020.6.8 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (2022.10.31)\n",
      "Requirement already satisfied: dill<0.3.2,>=0.3.1.1 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (0.3.1.1)\n",
      "Requirement already satisfied: orjson<4.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (3.8.10)\n",
      "Requirement already satisfied: pymongo<4.0.0,>=3.8.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (3.13.0)\n",
      "Requirement already satisfied: pytz>=2018.3 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (2023.3)\n",
      "Requirement already satisfied: fastavro<2,>=0.23.6 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (1.7.3)\n",
      "Requirement already satisfied: cloudpickle~=2.2.1 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (2.2.1)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.24.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (2.28.2)\n",
      "Requirement already satisfied: crcmod<2.0,>=1.7 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (1.7)\n",
      "Requirement already satisfied: typing-extensions>=3.7.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (4.5.0)\n",
      "Requirement already satisfied: zstandard<1,>=0.18.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (0.18.0)\n",
      "Requirement already satisfied: proto-plus<2,>=1.7.1 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (1.22.2)\n",
      "Requirement already satisfied: grpcio!=1.48.0,<2,>=1.33.1 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (1.53.0)\n",
      "Requirement already satisfied: httplib2<0.22.0,>=0.8 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (0.21.0)\n",
      "Requirement already satisfied: google-cloud-pubsub<3,>=2.1.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (2.16.0)\n",
      "Requirement already satisfied: google-cloud-core<3,>=0.28.1 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (2.3.2)\n",
      "Requirement already satisfied: google-cloud-vision<4,>=2 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (3.4.1)\n",
      "Requirement already satisfied: google-auth-httplib2<0.2.0,>=0.1.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (0.1.0)\n",
      "Requirement already satisfied: google-cloud-datastore<2,>=1.8.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (1.15.5)\n",
      "Requirement already satisfied: google-cloud-recommendations-ai<0.8.0,>=0.1.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (0.7.1)\n",
      "Requirement already satisfied: google-cloud-spanner<4,>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (3.30.0)\n",
      "Requirement already satisfied: google-cloud-language<2,>=1.3.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (1.3.2)\n",
      "Collecting cachetools<5,>=3.1.0\n",
      "  Downloading cachetools-4.2.4-py3-none-any.whl (10 kB)\n",
      "Requirement already satisfied: google-cloud-pubsublite<2,>=1.2.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (1.7.0)\n",
      "Requirement already satisfied: google-cloud-bigquery-storage<2.17,>=2.6.3 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (2.16.2)\n",
      "Requirement already satisfied: google-cloud-bigquery<4,>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (3.9.0)\n",
      "Requirement already satisfied: google-auth<3,>=1.18.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (2.17.2)\n",
      "Requirement already satisfied: google-cloud-bigtable<2,>=0.31.1 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (1.7.3)\n",
      "Requirement already satisfied: google-apitools<0.5.32,>=0.5.31 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (0.5.31)\n",
      "Requirement already satisfied: google-cloud-dlp<4,>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (3.12.1)\n",
      "Requirement already satisfied: google-cloud-videointelligence<2,>=1.8.0 in /opt/conda/lib/python3.7/site-packages (from apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (1.16.3)\n",
      "Requirement already satisfied: pyparsing>=2.1.4 in /opt/conda/lib/python3.7/site-packages (from pydot<2,>=1.2->tensorflow-transform==1.6.0) (3.0.9)\n",
      "Requirement already satisfied: keras-preprocessing>=1.1.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (1.1.2)\n",
      "Collecting keras<2.8,>=2.7.0rc0\n",
      "  Downloading keras-2.7.0-py2.py3-none-any.whl (1.3 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m65.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: tensorflow-io-gcs-filesystem>=0.21.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (0.25.0)\n",
      "Requirement already satisfied: wheel<1.0,>=0.32.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (0.40.0)\n",
      "Collecting tensorflow-estimator<2.8,~=2.7.0rc0\n",
      "  Downloading tensorflow_estimator-2.7.0-py2.py3-none-any.whl (463 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m463.1/463.1 kB\u001b[0m \u001b[31m40.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: termcolor>=1.1.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (2.2.0)\n",
      "Collecting flatbuffers<3.0,>=1.12\n",
      "  Downloading flatbuffers-2.0.7-py2.py3-none-any.whl (26 kB)\n",
      "Collecting protobuf<4,>=3.13\n",
      "  Downloading protobuf-3.19.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m57.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting gast<0.5.0,>=0.2.1\n",
      "  Downloading gast-0.4.0-py3-none-any.whl (9.8 kB)\n",
      "Requirement already satisfied: h5py>=2.9.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (3.8.0)\n",
      "Requirement already satisfied: wrapt>=1.11.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (1.15.0)\n",
      "Requirement already satisfied: libclang>=9.0.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (16.0.0)\n",
      "Requirement already satisfied: tensorboard~=2.6 in /opt/conda/lib/python3.7/site-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (2.8.0)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /opt/conda/lib/python3.7/site-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (3.3.0)\n",
      "Requirement already satisfied: six>=1.12.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (1.16.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (0.2.0)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (1.6.3)\n",
      "Requirement already satisfied: googleapis-common-protos<2,>=1.52.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow-metadata<1.7.0,>=1.6.0->tensorflow-transform==1.6.0) (1.59.0)\n",
      "Requirement already satisfied: google-api-python-client<2,>=1.7.11 in /opt/conda/lib/python3.7/site-packages (from tfx-bsl<1.7.0,>=1.6.0->tensorflow-transform==1.6.0) (1.8.0)\n",
      "Requirement already satisfied: tensorflow-serving-api!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<3,>=1.15 in /opt/conda/lib/python3.7/site-packages (from tfx-bsl<1.7.0,>=1.6.0->tensorflow-transform==1.6.0) (2.8.4)\n",
      "Requirement already satisfied: pandas<2,>=1.0 in /opt/conda/lib/python3.7/site-packages (from tfx-bsl<1.7.0,>=1.6.0->tensorflow-transform==1.6.0) (1.3.5)\n",
      "Collecting google-api-core<2dev,>=1.13.0\n",
      "  Downloading google_api_core-1.34.0-py3-none-any.whl (120 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m120.2/120.2 kB\u001b[0m \u001b[31m16.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: uritemplate<4dev,>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from google-api-python-client<2,>=1.7.11->tfx-bsl<1.7.0,>=1.6.0->tensorflow-transform==1.6.0) (3.0.1)\n",
      "Requirement already satisfied: oauth2client>=1.4.12 in /opt/conda/lib/python3.7/site-packages (from google-apitools<0.5.32,>=0.5.31->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (4.1.3)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.18.0->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (4.9)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.18.0->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (0.2.8)\n",
      "Requirement already satisfied: google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5 in /opt/conda/lib/python3.7/site-packages (from google-cloud-bigquery<4,>=1.6.0->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (2.10.1)\n",
      "Requirement already satisfied: packaging>=20.0.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-bigquery<4,>=1.6.0->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (23.0)\n",
      "Requirement already satisfied: google-resumable-media<3.0dev,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-bigquery<4,>=1.6.0->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (2.4.1)\n",
      "Requirement already satisfied: grpc-google-iam-v1<0.13dev,>=0.12.3 in /opt/conda/lib/python3.7/site-packages (from google-cloud-bigtable<2,>=0.31.1->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (0.12.6)\n",
      "Collecting google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5\n",
      "  Downloading google_api_core-2.11.0-py3-none-any.whl (120 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m120.3/120.3 kB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: grpcio-status>=1.33.2 in /opt/conda/lib/python3.7/site-packages (from google-cloud-pubsub<3,>=2.1.0->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (1.48.2)\n",
      "Requirement already satisfied: overrides<7.0.0,>=6.0.1 in /opt/conda/lib/python3.7/site-packages (from google-cloud-pubsublite<2,>=1.2.0->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (6.5.0)\n",
      "Requirement already satisfied: sqlparse>=0.3.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-spanner<4,>=3.0.0->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (0.4.3)\n",
      "Requirement already satisfied: docopt in /opt/conda/lib/python3.7/site-packages (from hdfs<3.0.0,>=2.1.0->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (0.6.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0,>=2.24.0->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (2.1.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0,>=2.24.0->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (2022.12.7)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0,>=2.24.0->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (1.26.15)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0,>=2.24.0->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (3.4)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard~=2.6->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (0.6.1)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard~=2.6->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (1.8.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /opt/conda/lib/python3.7/site-packages (from tensorboard~=2.6->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (0.4.6)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /opt/conda/lib/python3.7/site-packages (from tensorboard~=2.6->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (2.1.2)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /opt/conda/lib/python3.7/site-packages (from tensorboard~=2.6->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (3.4.3)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard~=2.6->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (67.6.1)\n",
      "Collecting tensorflow-serving-api!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<3,>=1.15\n",
      "  Downloading tensorflow_serving_api-2.11.1-py2.py3-none-any.whl (37 kB)\n",
      "  Downloading tensorflow_serving_api-2.11.0-py2.py3-none-any.whl (37 kB)\n",
      "  Downloading tensorflow_serving_api-2.10.1-py2.py3-none-any.whl (37 kB)\n",
      "  Downloading tensorflow_serving_api-2.10.0-py2.py3-none-any.whl (37 kB)\n",
      "  Downloading tensorflow_serving_api-2.9.3-py2.py3-none-any.whl (37 kB)\n",
      "  Downloading tensorflow_serving_api-2.9.2-py2.py3-none-any.whl (37 kB)\n",
      "  Downloading tensorflow_serving_api-2.9.1-py2.py3-none-any.whl (37 kB)\n",
      "  Downloading tensorflow_serving_api-2.9.0-py2.py3-none-any.whl (37 kB)\n",
      "  Downloading tensorflow_serving_api-2.8.3-py2.py3-none-any.whl (37 kB)\n",
      "  Downloading tensorflow_serving_api-2.8.2-py2.py3-none-any.whl (37 kB)\n",
      "  Downloading tensorflow_serving_api-2.8.0-py2.py3-none-any.whl (37 kB)\n",
      "  Downloading tensorflow_serving_api-2.7.4-py2.py3-none-any.whl (37 kB)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /opt/conda/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (1.3.1)\n",
      "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /opt/conda/lib/python3.7/site-packages (from google-resumable-media<3.0dev,>=0.6.0->google-cloud-bigquery<4,>=1.6.0->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (1.5.0)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in /opt/conda/lib/python3.7/site-packages (from markdown>=2.6.8->tensorboard~=2.6->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (6.0.1)\n",
      "Requirement already satisfied: pyasn1>=0.1.7 in /opt/conda/lib/python3.7/site-packages (from oauth2client>=1.4.12->google-apitools<0.5.32,>=0.5.31->apache-beam[gcp]<3,>=2.35->tensorflow-transform==1.6.0) (0.4.8)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard~=2.6->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (3.15.0)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,<2.8,>=1.15.5->tensorflow-transform==1.6.0) (3.2.2)\n",
      "Installing collected packages: tensorflow-estimator, keras, flatbuffers, pyarrow, protobuf, gast, cachetools, tensorflow-metadata, google-api-core, tensorflow, tensorflow-serving-api, tfx-bsl, tensorflow-transform\n",
      "  Attempting uninstall: tensorflow-estimator\n",
      "    Found existing installation: tensorflow-estimator 2.8.0\n",
      "    Uninstalling tensorflow-estimator-2.8.0:\n",
      "      Successfully uninstalled tensorflow-estimator-2.8.0\n",
      "  Attempting uninstall: keras\n",
      "    Found existing installation: keras 2.8.0\n",
      "    Uninstalling keras-2.8.0:\n",
      "      Successfully uninstalled keras-2.8.0\n",
      "  Attempting uninstall: flatbuffers\n",
      "    Found existing installation: flatbuffers 23.3.3\n",
      "    Uninstalling flatbuffers-23.3.3:\n",
      "      Successfully uninstalled flatbuffers-23.3.3\n",
      "  Attempting uninstall: pyarrow\n",
      "    Found existing installation: pyarrow 9.0.0\n",
      "    Uninstalling pyarrow-9.0.0:\n",
      "      Successfully uninstalled pyarrow-9.0.0\n",
      "  Attempting uninstall: protobuf\n",
      "    Found existing installation: protobuf 3.20.1\n",
      "    Uninstalling protobuf-3.20.1:\n",
      "      Successfully uninstalled protobuf-3.20.1\n",
      "  Attempting uninstall: gast\n",
      "    Found existing installation: gast 0.5.3\n",
      "    Uninstalling gast-0.5.3:\n",
      "      Successfully uninstalled gast-0.5.3\n",
      "  Attempting uninstall: cachetools\n",
      "    Found existing installation: cachetools 5.3.0\n",
      "    Uninstalling cachetools-5.3.0:\n",
      "      Successfully uninstalled cachetools-5.3.0\n",
      "  Attempting uninstall: tensorflow-metadata\n",
      "    Found existing installation: tensorflow-metadata 0.14.0\n",
      "    Uninstalling tensorflow-metadata-0.14.0:\n",
      "      Successfully uninstalled tensorflow-metadata-0.14.0\n",
      "  Attempting uninstall: google-api-core\n",
      "    Found existing installation: google-api-core 2.10.1\n",
      "    Uninstalling google-api-core-2.10.1:\n",
      "      Successfully uninstalled google-api-core-2.10.1\n",
      "  Attempting uninstall: tensorflow\n",
      "    Found existing installation: tensorflow 2.8.4\n",
      "    Uninstalling tensorflow-2.8.4:\n",
      "      Successfully uninstalled tensorflow-2.8.4\n",
      "  Attempting uninstall: tensorflow-serving-api\n",
      "    Found existing installation: tensorflow-serving-api 2.8.4\n",
      "    Uninstalling tensorflow-serving-api-2.8.4:\n",
      "      Successfully uninstalled tensorflow-serving-api-2.8.4\n",
      "  Attempting uninstall: tensorflow-transform\n",
      "    Found existing installation: tensorflow-transform 0.14.0\n",
      "    Uninstalling tensorflow-transform-0.14.0:\n",
      "      Successfully uninstalled tensorflow-transform-0.14.0\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "google-cloud-aiplatform 1.23.0 requires packaging<22.0.0dev,>=14.3, but you have packaging 23.0 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed cachetools-4.2.4 flatbuffers-2.0.7 gast-0.4.0 google-api-core-1.34.0 keras-2.7.0 protobuf-3.19.6 pyarrow-5.0.0 tensorflow-2.7.4 tensorflow-estimator-2.7.0 tensorflow-metadata-1.6.0 tensorflow-serving-api-2.7.4 tensorflow-transform-1.6.0 tfx-bsl-1.6.0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!sudo -i pip install tensorflow-transform==1.6.0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29be7b0b-52c2-42be-8d3d-691298eabc97",
   "metadata": {},
   "source": [
    "You can ingnore the dependency resolver errors. Confirm the final message starting with \"Successfully installed ...\"\n",
    "\n",
    "**Now you have to restart kernel from the menu bar: \"Kernel\" -> \"Restart Kernel\".**\n",
    "\n",
    "After restarting the kernel, you can resume the code execution from the next cell."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bdaa4984-6cee-4d54-a588-a7b6f9f9bcb6",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Confirm the installed packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7ad43b57-7e4b-410e-b354-5dab0c439b89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apache-beam                            2.46.0\n",
      "tensorflow                             2.7.4\n",
      "tensorflow-cloud                       0.1.16\n",
      "tensorflow-datasets                    4.8.2\n",
      "tensorflow-estimator                   2.7.0\n",
      "tensorflow-hub                         0.13.0\n",
      "tensorflow-io                          0.25.0\n",
      "tensorflow-io-gcs-filesystem           0.25.0\n",
      "tensorflow-metadata                    1.6.0\n",
      "tensorflow-probability                 0.19.0\n",
      "tensorflow-serving-api                 2.7.4\n",
      "tensorflow-transform                   1.6.0\n"
     ]
    }
   ],
   "source": [
    "!pip list | grep -E '(tensorflow|beam)'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f472115-8b24-4c48-af9a-8c08627d50f6",
   "metadata": {},
   "source": [
    "### Create setup.py to install packages to Dataflow containers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5475f8c-e551-4281-bd76-9c3fee920474",
   "metadata": {},
   "source": [
    "This is used to install additional packages to Dataflow worker containers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f021734a-0439-4a5a-87e3-d9209ba15ba1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing setup.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile setup.py\n",
    "import setuptools\n",
    "\n",
    "setuptools.setup(\n",
    "    install_requires=['tensorflow-transform==1.6.0'],\n",
    "    packages=setuptools.find_packages(),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0db8027-1722-43ce-a0ac-b12aec7250d5",
   "metadata": {},
   "source": [
    "### Set global flags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a81e0009-101a-4fb6-98c1-6c8181d6ab31",
   "metadata": {},
   "outputs": [],
   "source": [
    "PROJECT = 'your-project'\n",
    "BUCKET = 'your-project-babyweight'\n",
    "REGION = 'us-central1'\n",
    "ROOT_DIR = 'babyweight_tft'\n",
    "\n",
    "RUN_LOCAL = False # if True, the DirectRunner is used, else DataflowRunner\n",
    "DATA_SIZE = 10000 # number of records to be retrieved from BigQuery"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fcec29c0-1d9e-4024-97a0-a6d5eaf188e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['PROJECT'] = PROJECT\n",
    "os.environ['BUCKET'] = BUCKET\n",
    "os.environ['REGION'] = REGION\n",
    "os.environ['ROOT_DIR'] = ROOT_DIR\n",
    "os.environ['RUN_LOCAL'] = str(RUN_LOCAL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b668941a-98ab-4857-9e3d-8398adfd5391",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Updated property [core/project].\n",
      "Updated property [compute/region].\n",
      "Creating gs://etsuji-tft1-babyweight/...\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "gcloud config set project $PROJECT\n",
    "gcloud config set compute/region $REGION\n",
    "if ! gcloud storage ls | grep -q gs://${BUCKET}/; then\n",    "    gcloud storage buckets create --location=${REGION} gs://${BUCKET}\n",    "fi"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bafc6791-37bd-48be-9b7f-9219bdf742fc",
   "metadata": {},
   "source": [
    "### Import required packages and modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d7d7e950-1ca2-44c4-8e35-1919ba29078a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import os\n",
    "\n",
    "import apache_beam as beam\n",
    "import tensorflow.compat.v2 as tf\n",
    "import tensorflow_transform as tft\n",
    "\n",
    "import tensorflow_transform.beam as tft_beam\n",
    "from tensorflow_transform.tf_metadata import dataset_metadata\n",
    "from tensorflow_transform.tf_metadata import schema_utils\n",
    "from tensorflow_transform.beam.tft_beam_io import transform_fn_io\n",
    "from tfx_bsl.coders import example_coder"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ab0e492-4e81-464c-9881-ac3b4b26101e",
   "metadata": {},
   "source": [
    "### Define raw input data and their metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "15ae75e0-ff15-4c34-88b1-7a93cab9f1d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "CATEGORICAL_FEATURE_NAMES = ['is_male', 'mother_race']\n",
    "NUMERIC_FEATURE_NAMES = ['mother_age', 'plurality', 'gestation_weeks']\n",
    "TARGET_FEATURE_NAME = 'weight_pounds'\n",
    "\n",
    "def create_raw_metadata():  \n",
    "\n",
    "    feature_spec = dict(\n",
    "        [(name, tf.io.FixedLenFeature([], tf.string)) for name in CATEGORICAL_FEATURE_NAMES] +\n",
    "        [(name, tf.io.FixedLenFeature([], tf.float32)) for name in NUMERIC_FEATURE_NAMES] +\n",
    "        [(TARGET_FEATURE_NAME, tf.io.FixedLenFeature([], tf.float32))])\n",
    "\n",
    "    raw_metadata = dataset_metadata.DatasetMetadata(\n",
    "        schema_utils.schema_from_feature_spec(feature_spec))\n",
    "    \n",
    "    return raw_metadata"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c832e6ae-a910-490c-8c0c-0b7c6a9af577",
   "metadata": {},
   "source": [
    "The metadata contains feature schema in the protobuf format as below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "34d17003-4fc5-4b5b-b21d-a0b3f4a22ac3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "feature {\n",
       "  name: \"gestation_weeks\"\n",
       "  type: FLOAT\n",
       "  presence {\n",
       "    min_fraction: 1.0\n",
       "  }\n",
       "  shape {\n",
       "  }\n",
       "}\n",
       "feature {\n",
       "  name: \"is_male\"\n",
       "  type: BYTES\n",
       "  presence {\n",
       "    min_fraction: 1.0\n",
       "  }\n",
       "  shape {\n",
       "  }\n",
       "}\n",
       "feature {\n",
       "  name: \"mother_age\"\n",
       "  type: FLOAT\n",
       "  presence {\n",
       "    min_fraction: 1.0\n",
       "  }\n",
       "  shape {\n",
       "  }\n",
       "}\n",
       "feature {\n",
       "  name: \"mother_race\"\n",
       "  type: BYTES\n",
       "  presence {\n",
       "    min_fraction: 1.0\n",
       "  }\n",
       "  shape {\n",
       "  }\n",
       "}\n",
       "feature {\n",
       "  name: \"plurality\"\n",
       "  type: FLOAT\n",
       "  presence {\n",
       "    min_fraction: 1.0\n",
       "  }\n",
       "  shape {\n",
       "  }\n",
       "}\n",
       "feature {\n",
       "  name: \"weight_pounds\"\n",
       "  type: FLOAT\n",
       "  presence {\n",
       "    min_fraction: 1.0\n",
       "  }\n",
       "  shape {\n",
       "  }\n",
       "}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "create_raw_metadata().schema"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e12bec8e-1cc5-416f-b57a-92facbe68c57",
   "metadata": {},
   "source": [
    "### Define source query and source cleanup function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0c422ff3-df95-4807-8ab0-2888efb90795",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_source_query(step, data_size):\n",
    "    \n",
    "    train_size = data_size * 0.7\n",
    "    eval_size = data_size * 0.3\n",
    "    \n",
    "    query = \"\"\"\n",
    "    SELECT\n",
    "      ROUND(weight_pounds,1) AS weight_pounds,\n",
    "      is_male,\n",
    "      mother_age,\n",
    "      mother_race,\n",
    "      plurality,\n",
    "      gestation_weeks,\n",
    "      FARM_FINGERPRINT( \n",
    "        CONCAT(\n",
    "          COALESCE(CAST(weight_pounds AS STRING), 'NA'),\n",
    "          COALESCE(CAST(is_male AS STRING),'NA'),\n",
    "          COALESCE(CAST(mother_age AS STRING),'NA'),\n",
    "          COALESCE(CAST(mother_race AS STRING),'NA'),\n",
    "          COALESCE(CAST(plurality AS STRING), 'NA'),\n",
    "          COALESCE(CAST(gestation_weeks AS STRING),'NA')\n",
    "          )\n",
    "        ) AS key\n",
    "        FROM\n",
    "          publicdata.samples.natality\n",
    "        WHERE year > 2000\n",
    "        AND weight_pounds > 0\n",
    "        AND mother_age > 0\n",
    "        AND plurality > 0\n",
    "        AND gestation_weeks > 0\n",
    "        AND month > 0\n",
    "    \"\"\"\n",
    "    \n",
    "    if step == 'train':\n",
    "        source_query = 'SELECT * FROM ({}) WHERE MOD(key, 100) < 70 LIMIT {}'.format(query, int(train_size))\n",
    "    else:\n",
    "        source_query = 'SELECT * FROM ({}) WHERE MOD(key, 100) >= 70 LIMIT {}'.format(query, int(eval_size))\n",
    "    \n",
    "    return source_query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "58ec77ef-0c60-4a0b-baca-f90409c6ca77",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prep_bq_row(bq_row):\n",
    "\n",
    "    # modify opaque numeric race code into human-readable data\n",
    "    races = dict(zip([1,2,3,4,5,6,7,18,28,39,48],\n",
    "                     ['White', 'Black', 'American Indian', 'Chinese', \n",
    "                      'Japanese', 'Hawaiian', 'Filipino',\n",
    "                      'Asian Indian', 'Korean', 'Samaon', 'Vietnamese']))\n",
    "    result = {} \n",
    "    \n",
    "    for feature_name in bq_row.keys():\n",
    "        if isinstance(bq_row[feature_name], bool):\n",
    "            result[feature_name] = str(bq_row[feature_name])\n",
    "        else:\n",
    "            result[feature_name] = bq_row[feature_name]\n",
    "\n",
    "    if 'mother_race' in bq_row and bq_row['mother_race'] in races:\n",
    "        result['mother_race'] = races[bq_row['mother_race']]\n",
    "    else:\n",
    "        result['mother_race'] = 'Unknown'\n",
    "\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0d1ca02-d66b-4a90-bd00-2a17311dd1ad",
   "metadata": {},
   "source": [
    "The output from the cleanup function `prep_bq_row` is used as raw input data."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f35c4d29-da6d-45e5-be85-7680f822340a",
   "metadata": {},
   "source": [
    "## 2. Define data transformation pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f53aeab-2053-4f31-b312-4de89ab7d0b9",
   "metadata": {},
   "source": [
    "First, we define component functions in the pipeline."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3b76ff2-93e5-48ff-953f-a24ddd81d8c9",
   "metadata": {},
   "source": [
    "### Read and clean from source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ee9850b3-9627-4982-a075-54a3209986a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_from_bq(pipeline, step, data_size):\n",
    "    \n",
    "    source_query = get_source_query(step, data_size)\n",
    "    raw_data = (\n",
    "        pipeline\n",
    "        | '{} - Read Data from BigQuery'.format(step) >> beam.io.Read(\n",
    "            beam.io.BigQuerySource(query=source_query, use_standard_sql=True))\n",
    "        | '{} - Clean up Data'.format(step) >> beam.Map(prep_bq_row)\n",
    "    )\n",
    "\n",
    "    # Assosiate the metadata to the raw input data. The metadata are used for the transformation.\n",
    "    raw_metadata = create_raw_metadata()\n",
    "    raw_dataset = (raw_data, raw_metadata)\n",
    "    return raw_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cdeb3fd9-5067-4bd0-a6d2-0165aa0af2aa",
   "metadata": {},
   "source": [
    "### tf.Transform preprocess_fn"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53b9f997-2ff6-4e6f-b160-dd942f75b40d",
   "metadata": {},
   "source": [
    "This function defines the data transformation against raw input data. This will be used by the data transformation pipeline, and also embeded in the exported model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cc48b270-c1b9-429a-acf9-b6ba3e407493",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_fn(input_features):\n",
    "\n",
    "    # explicitly import packages here as the apache beam fails to serialize them from the global context.\n",
    "    import tensorflow.compat.v2 as tf\n",
    "    import tensorflow_transform as tft\n",
    "\n",
    "    output_features = {}\n",
    "\n",
    "    # target feature\n",
    "    output_features['weight_pounds'] = input_features['weight_pounds']\n",
    "\n",
    "    # normalisation\n",
    "    output_features['mother_age_normalized'] = tft.scale_to_z_score(input_features['mother_age'])\n",
    "    \n",
    "    # scaling\n",
    "    output_features['gestation_weeks_scaled'] =  tft.scale_to_0_1(input_features['gestation_weeks'])\n",
    "    \n",
    "    # bucketisation based on quantiles\n",
    "    output_features['mother_age_bucketized'] = tft.bucketize(input_features['mother_age'], num_buckets=5)\n",
    "    \n",
    "    # you can compute new features based on custom formulas\n",
    "    output_features['mother_age_log'] = tf.math.log(input_features['mother_age'])\n",
    "    \n",
    "    # or create flags/indicators\n",
    "    is_multiple = tf.as_string(input_features['plurality'] > tf.constant(1.0))\n",
    "    \n",
    "    # convert categorical features to indexed vocab\n",
    "    output_features['mother_race_index'] = tft.compute_and_apply_vocabulary(input_features['mother_race'], vocab_filename='mother_race')\n",
    "    output_features['is_male_index'] = tft.compute_and_apply_vocabulary(input_features['is_male'], vocab_filename='is_male')\n",
    "    output_features['is_multiple_index'] = tft.compute_and_apply_vocabulary(is_multiple, vocab_filename='is_multiple')\n",
    "    \n",
    "    return output_features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "449e0d4a-c5c1-4ff3-88d7-1c12f1e2fc60",
   "metadata": {},
   "source": [
    "### Analyze and transform"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ba346d9-64c7-41e2-8d64-44fbf87c3156",
   "metadata": {},
   "source": [
    "This is applied to the training dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "67798d22-11fa-4c54-99f5-e22aeef9efbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyze_and_transform(raw_dataset, step):\n",
    "    \n",
    "    transformed_dataset, transform_fn = (\n",
    "        raw_dataset \n",
    "        | '{} - Analyze & Transform'.format(step) >> tft_beam.AnalyzeAndTransformDataset(\n",
    "            preprocess_fn, output_record_batches=True)\n",
    "    )\n",
    "    \n",
    "    return transformed_dataset, transform_fn"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc0327c9-0d7d-42a6-9cfe-0e3fbf73d139",
   "metadata": {},
   "source": [
    "### Transform"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8420438-cb01-4d43-9bcf-8fc7f83728b0",
   "metadata": {},
   "source": [
    "This is applied to the evaluation dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "795b76f5-b538-4b9a-aa92-65e8b267f6ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform(raw_dataset, transform_fn, step):\n",
    "    \n",
    "    transformed_dataset = (\n",
    "        (raw_dataset, transform_fn) \n",
    "        | '{} - Transform'.format(step) >> tft_beam.TransformDataset(output_record_batches=True)\n",
    "    )\n",
    "    \n",
    "    return transformed_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de612fca-4c2d-4192-adbb-74548b55965e",
   "metadata": {},
   "source": [
    "### Write tfrecords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "62cbee0b-109e-4e93-a42a-5f42573f3394",
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_tfrecords(transformed_dataset, location, step):\n",
    "    from tfx_bsl.coders import example_coder\n",
    "\n",
    "    transformed_data, transformed_metadata = transformed_dataset\n",
    "    (\n",
    "        transformed_data\n",
    "        | '{} - Encode Transformed Data'.format(step) >> beam.FlatMapTuple(\n",
    "                            lambda batch, _: example_coder.RecordBatchToExamples(batch))\n",
    "        | '{} - Write Transformed Data'.format(step) >> beam.io.WriteToTFRecord(\n",
    "                            file_path_prefix=os.path.join(location,'{}'.format(step)),\n",
    "                            file_name_suffix='.tfrecords')\n",
    "    )  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7663fb1-2280-40f1-b114-b333594fc2b8",
   "metadata": {},
   "source": [
    "### Write text records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f2978df4-afa1-4410-9876-b6e2e4e4b40d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_text(dataset, location, step):\n",
    "    \n",
    "    data, _ = dataset\n",
    "    (\n",
    "        data \n",
    "        | '{} - WriteData'.format(step) >> beam.io.WriteToText(\n",
    "            file_path_prefix=os.path.join(location,'{}'.format(step)),\n",
    "            file_name_suffix='.txt')\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d719d45-e5c4-4867-bbb8-29629df4872d",
   "metadata": {},
   "source": [
    "### Write transformation artefacts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "167da7a2-00a1-446e-a420-2bda7229b077",
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_transform_artefacts(transform_fn, location):\n",
    "    \n",
    "    (\n",
    "        transform_fn \n",
    "        | 'Write Transform Artefacts' >> transform_fn_io.WriteTransformFn(location)\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8df0db43-3fd8-4681-b772-82eb236c2833",
   "metadata": {},
   "source": [
    "Now we can construct a pipeline by combining components."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25384b1d-0f1e-4980-bd69-3486f69b8c1b",
   "metadata": {},
   "source": [
    "### Construct data transformation pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ec745a34-7031-491f-8e70-a1557af0fbdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_transformation_pipeline(args):\n",
    "    \n",
    "    pipeline_options = beam.pipeline.PipelineOptions(flags=[], **args)\n",
    "    \n",
    "    runner = args['runner']\n",
    "    data_size = args['data_size']\n",
    "    transformed_data_location = args['transformed_data_location']\n",
    "    transform_artefact_location = args['transform_artefact_location']\n",
    "    temporary_dir = args['temporary_dir']\n",
    "    debug = args['debug']\n",
    "    \n",
    "    print(\"Sample data size: {}\".format(data_size))\n",
    "    print(\"Sink transformed data files location: {}\".format(transformed_data_location))\n",
    "    print(\"Sink transform artefact location: {}\".format(transform_artefact_location))\n",
    "    print(\"Temporary directory: {}\".format(temporary_dir))\n",
    "    print(\"Runner: {}\".format(runner))\n",
    "    print(\"Debug enabled: {}\".format(debug))\n",
    "\n",
    "    with beam.Pipeline(runner, options=pipeline_options) as pipeline:\n",
    "        with tft_beam.Context(temporary_dir):\n",
    "            \n",
    "            # Preprocess train data\n",
    "            step = 'train'\n",
    "            # Read raw train data from BQ\n",
    "            raw_train_dataset = read_from_bq(pipeline, step, data_size)\n",
    "            # Analyze and transform raw_train_dataset \n",
    "            transformed_train_dataset, transform_fn = analyze_and_transform(raw_train_dataset, step)\n",
    "            # Write transformed train data to sink as tfrecords\n",
    "            write_tfrecords(transformed_train_dataset, transformed_data_location, step)\n",
    "            \n",
    "            # Preprocess evaluation data\n",
    "            step = 'eval'\n",
    "            # Read raw eval data from BQ\n",
    "            raw_eval_dataset = read_from_bq(pipeline, step, data_size)\n",
    "            # Transform eval data based on produced transform_fn\n",
    "            transformed_eval_dataset = transform(raw_eval_dataset, transform_fn, step)\n",
    "            # Write transformed eval data to sink as tfrecords\n",
    "            write_tfrecords(transformed_eval_dataset, transformed_data_location, step)\n",
    "            \n",
    "            # Write transformation artefacts \n",
    "            write_transform_artefacts(transform_fn, transform_artefact_location)\n",
    "\n",
    "            # (Optional) for debugging, write transformed data as text \n",
    "            step = 'debug'\n",
    "            # Wwrite transformed train data as text if debug enabled\n",
    "            if debug == True:\n",
    "                write_text(transformed_train_dataset, transformed_data_location, step)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cab0a5d2-367a-46c9-adfc-47d2f41a699e",
   "metadata": {},
   "source": [
    "## 3. Execute transformation pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f91c9768-d624-442d-9c05-9a040ca527cd",
   "metadata": {},
   "source": [
    "### Set pipeline parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "27db9263-51c8-4484-891b-4910c3090f88",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "OUTPUT_DIR = \"gs://{}/{}\".format(BUCKET, ROOT_DIR)\n",
    "TRANSFORM_ARTEFACTS_DIR = os.path.join(OUTPUT_DIR, 'transform')\n",
    "TRANSFORMED_DATA_DIR = os.path.join(OUTPUT_DIR, 'transformed')\n",
    "TEMP_DIR = os.path.join(OUTPUT_DIR, 'tmp')\n",
    "\n",
    "if RUN_LOCAL:\n",
    "    runner = 'DirectRunner'\n",
    "else:\n",
    "    runner = 'DataflowRunner'\n",
    "\n",
    "job_name = 'preprocess-babweight-data-tft-{}'.format(datetime.utcnow().strftime('%y%m%d-%H%M%S'))\n",
    "\n",
    "args = {\n",
    "    'job_name': job_name,\n",
    "    'runner': runner,\n",
    "    'data_size': DATA_SIZE,\n",
    "    'transformed_data_location':  TRANSFORMED_DATA_DIR,\n",
    "    'transform_artefact_location':  TRANSFORM_ARTEFACTS_DIR,\n",
    "    'temporary_dir': TEMP_DIR,\n",
    "    'debug': False,\n",
    "    \n",
    "    'project': PROJECT,\n",
    "    'region': REGION,\n",
    "    'staging_location': os.path.join(OUTPUT_DIR, 'staging'),\n",
    "    'temp_location': os.path.join(OUTPUT_DIR, 'tmp'),\n",
    "    'worker_machine_type': 'n1-standard-1',\n",
    "    'max_num_workers': 3,\n",
    "    'setup_file': './setup.py', # requirements_file doesn't work as tft provides only wheel pkg.\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c615e594-8263-4a51-91c2-71b79f3360c4",
   "metadata": {},
   "source": [
    "### Run pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c36ecf3d-4265-46ff-86db-02508079085b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Launching DataflowRunner job preprocess-babweight-data-tft-230419-024433 ... hang on\n",
      "\n",
      "Sample data size: 10000\n",
      "Sink transformed data files location: gs://etsuji-tft1-babyweight/babyweight_tft/transformed\n",
      "Sink transform artefact location: gs://etsuji-tft1-babyweight/babyweight_tft/transform\n",
      "Temporary directory: gs://etsuji-tft1-babyweight/babyweight_tft/tmp\n",
      "Runner: DataflowRunner\n",
      "Debug enabled: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:8: BeamDeprecationWarning: BigQuerySource is deprecated since 2.25.0. Use ReadFromBigQuery instead.\n",
      "  \n",
      "WARNING:apache_beam.runners.interactive.interactive_environment:Dependencies required for Interactive Beam PCollection visualization are not available, please use: `pip install apache-beam[interactive]` to install necessary dependencies to enable all data visualization features.\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "        if (typeof window.interactive_beam_jquery == 'undefined') {\n",
       "          var jqueryScript = document.createElement('script');\n",
       "          jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n",
       "          jqueryScript.type = 'text/javascript';\n",
       "          jqueryScript.onload = function() {\n",
       "            var datatableScript = document.createElement('script');\n",
       "            datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n",
       "            datatableScript.type = 'text/javascript';\n",
       "            datatableScript.onload = function() {\n",
       "              window.interactive_beam_jquery = jQuery.noConflict(true);\n",
       "              window.interactive_beam_jquery(document).ready(function($){\n",
       "                \n",
       "              });\n",
       "            }\n",
       "            document.head.appendChild(datatableScript);\n",
       "          };\n",
       "          document.head.appendChild(jqueryScript);\n",
       "        } else {\n",
       "          window.interactive_beam_jquery(document).ready(function($){\n",
       "            \n",
       "          });\n",
       "        }"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/apache_beam/io/gcp/bigquery.py:2485: BeamDeprecationWarning: options is deprecated since First stable release. References to <pipeline>.options will not be supported\n",
      "  temp_location = pcoll.pipeline.options.view_as(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:You are passing instance dicts and DatasetMetadata to TFT which will not provide optimal performance. Consider following the TFT guide to upgrade to the TFXIO format (Apache Arrow RecordBatch).\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:You are passing instance dicts and DatasetMetadata to TFT which will not provide optimal performance. Consider following the TFT guide to upgrade to the TFXIO format (Apache Arrow RecordBatch).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_transform/tf_utils.py:289: Tensor.experimental_ref (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use ref() instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-04-19 02:44:39.859821: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/nccl2/lib:/usr/local/cuda/extras/CUPTI/lib64\n",
      "2023-04-19 02:44:39.859881: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2023-04-19 02:44:39.859912: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (tensorflow-2-8-20230419-113852): /proc/driver/nvidia/version does not exist\n",
      "2023-04-19 02:44:39.860393: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_transform/tf_utils.py:289: Tensor.experimental_ref (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use ref() instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:You are passing instance dicts and DatasetMetadata to TFT which will not provide optimal performance. Consider following the TFT guide to upgrade to the TFXIO format (Apache Arrow RecordBatch).\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:You are passing instance dicts and DatasetMetadata to TFT which will not provide optimal performance. Consider following the TFT guide to upgrade to the TFXIO format (Apache Arrow RecordBatch).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:You are passing instance dicts and DatasetMetadata to TFT which will not provide optimal performance. Consider following the TFT guide to upgrade to the TFXIO format (Apache Arrow RecordBatch).\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:You are passing instance dicts and DatasetMetadata to TFT which will not provide optimal performance. Consider following the TFT guide to upgrade to the TFXIO format (Apache Arrow RecordBatch).\n",
      "warning: check: missing required meta-data: name\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done!\n"
     ]
    }
   ],
   "source": [
    "try: \n",
    "    tf.io.gfile.rmtree(TRANSFORMED_DATA_DIR)\n",
    "    tf.io.gfile.rmtree(TRANSFORM_ARTEFACTS_DIR)\n",
    "    tf.io.gfile.rmtree(TEMP_DIR)\n",
    "    print('previous transformation files deleted!')\n",
    "except:\n",
    "    pass\n",
    "\n",
    "print('Launching {} job {} ... hang on'.format(runner, job_name))\n",
    "print('')\n",
    "run_transformation_pipeline(args)\n",
    "print('Done!')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a60e48a-7f40-4961-8d5a-29bbe579473f",
   "metadata": {},
   "source": [
    "### Explore the produced artefacts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4017c92b-4b2d-48da-bfac-c3c5ab8fad4e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "transformed data:\n",
      "gs://etsuji-tft1-babyweight/babyweight_tft/transformed/eval-00000-of-00001.tfrecords\n",
      "gs://etsuji-tft1-babyweight/babyweight_tft/transformed/train-00000-of-00001.tfrecords\n",
      "\n",
      "transformed metadata:\n",
      "gs://etsuji-tft1-babyweight/babyweight_tft/transform/transformed_metadata/\n",
      "gs://etsuji-tft1-babyweight/babyweight_tft/transform/transformed_metadata/asset_map\n",
      "gs://etsuji-tft1-babyweight/babyweight_tft/transform/transformed_metadata/schema.pbtxt\n",
      "\n",
      "transform artefact:\n",
      "gs://etsuji-tft1-babyweight/babyweight_tft/transform/transform_fn/\n",
      "gs://etsuji-tft1-babyweight/babyweight_tft/transform/transform_fn/saved_model.pb\n",
      "gs://etsuji-tft1-babyweight/babyweight_tft/transform/transform_fn/assets/\n",
      "gs://etsuji-tft1-babyweight/babyweight_tft/transform/transform_fn/variables/\n",
      "\n",
      "transform assets:\n",
      "gs://etsuji-tft1-babyweight/babyweight_tft/transform/transform_fn/assets/\n",
      "gs://etsuji-tft1-babyweight/babyweight_tft/transform/transform_fn/assets/is_male\n",
      "gs://etsuji-tft1-babyweight/babyweight_tft/transform/transform_fn/assets/is_multiple\n",
      "gs://etsuji-tft1-babyweight/babyweight_tft/transform/transform_fn/assets/mother_race\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "\n",
    "echo 'transformed data:' \n",
    "gcloud storage ls gs://${BUCKET}/${ROOT_DIR}/transformed \n",    "echo ''\n",
    "\n",
    "echo 'transformed metadata:'  \n",
    "gcloud storage ls gs://${BUCKET}/${ROOT_DIR}/transform/transformed_metadata \n",    "echo ''\n",
    "\n",
    "echo 'transform artefact:'   \n",
    "gcloud storage ls gs://${BUCKET}/${ROOT_DIR}/transform/transform_fn \n",    "echo ''\n",
    "\n",
    "echo 'transform assets:'\n",
    "gcloud storage ls gs://${BUCKET}/${ROOT_DIR}/transform/transform_fn/assets "
   ]
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "tf2-gpu.2-8.m107",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-8:m107"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
